#include "invertedIndex.hh"
#include "SplitTool.h"
#include <fstream>
#include <iostream>
#include <utility>
#include <cmath>
#include <map>
#include <vector>
#include "WebPage.hh"

//#include "src/WebPage.hh" //暂用，待删
//#include "src/PageLibPreprocessor.hh" //暂用，待删

using std::ifstream;
using std::cerr;
using std::cout;
using std::endl;
using std::istringstream;
using std::make_pair;
using std::map;
using std::ofstream;


InvertedIndex::InvertedIndex(const string &offsetLibPath, const string &netLibFilePath, const string &invertedIndexFilePath, 
                                const string &ZhStopWordsFilePath, const string &EngStopWordsFilePath, 
                                const map<int,pair<int,int>> &offsetLib, const vector<WebPage> &pageLib)
    : _offsetLibPath(offsetLibPath)
    , _netLibFilePath(netLibFilePath)
    , _invertedIndexFilePath(invertedIndexFilePath)
    , _ZhStopWordsFilePath(ZhStopWordsFilePath)
    , _EngStopWordsFilePath(EngStopWordsFilePath)
    , _offsetLib(offsetLib)
    , _pageLib(pageLib)
{}

InvertedIndex::~InvertedIndex()
{}

int InvertedIndex::readStopWords(){//停用词
    ifstream zhWordsFile(_ZhStopWordsFilePath);
    if (!zhWordsFile){
        cerr << "ifs open file " << _ZhStopWordsFilePath << " error!" << endl;
        return -1;
    }
    string word;
    while(getline(zhWordsFile, word)){ //一个词占一行
        _stopWords.push_back(word);
    }
    zhWordsFile.close();

    ifstream engWordsFile(_EngStopWordsFilePath);
    if (!engWordsFile){
        cerr << "ifs open file " << _EngStopWordsFilePath << " error!" << endl;
        return -1;
    }
    while(getline(engWordsFile, word)){ //一个词占一行
        _stopWords.push_back(word);
    }
    engWordsFile.close();

    //存入英文标点
    vector<string> engPunct = {"!", "(", ")", ",", ".", "?"};
    for(auto &elem: engPunct){
        _stopWords.push_back(elem);
    }

    return 0;
}

void InvertedIndex::filterStopWords(vector<string> &words){
    for(auto it = _stopWords.begin(); it != _stopWords.end(); ++it){
        for(auto it2 = words.begin(); it2 != words.end(); ){
            if(*it2 == *it){
                words.erase(it2);
            }
            else{
                ++it2;
            }
        }
    }
}
#if 0
int InvertedIndex::readOffsetFromFile(){
    _offsetVector.clear();
    ifstream ifs(_offsetLibPath);
    if(!ifs){
        cerr << "ifs open file " << _offsetLibPath << " error!" << endl;
        return -1;
    }

    string line;
    while(getline(ifs, line)){
        istringstream iss(line);
        Offset offset;
        iss >> offset.id;
        iss >> offset.start;
        iss >> offset.len;
        
        _offsetVector.push_back(offset);
    }
    ifs.close();
    return 0;
}
#endif
int InvertedIndex::readOffset(){
    _offsetVector.clear();
    Offset tempOffset;
    for(auto it = _offsetLib.begin(); it != _offsetLib.end(); ++it){
        tempOffset.id = it->first;
        tempOffset.start = it->second.first;
        tempOffset.len = it->second.second;
        _offsetVector.push_back(tempOffset);
    }
    return 0;
}

void InvertedIndex::printOffsetVector(){
    for(auto & elem: _offsetVector){
        cout << elem.id << " " 
        << elem.start << " " 
        << elem.len << endl;
    }
}

void InvertedIndex::printPageLibId(){
    for(auto it = _pageLib.begin(); it != _pageLib.end(); ++it){
        cout << "page id = " << it->_docId << endl;
    }
}

#if 0
int InvertedIndex::generateTableFromFile(){
    readOffsetFromFile();
    ifstream ifs(_netLibFilePath, std::ios::in);
    if(!ifs){
        cerr << "ifs open file " << _netLibFilePath << " error!" << endl;
        return -1;
    }

    int maxLen = 0;
    for(int i = 0; i < _offsetVector.size(); ++i){
        if(_offsetVector[i].len > maxLen){
            maxLen = _offsetVector[i].len;
        }
    }
    char* content = new char[maxLen + 1]();
    for(vector<Offset>::iterator it = _offsetVector.begin(); it != _offsetVector.end(); ++it){
        memset(content, 0, sizeof(content));
        ifs.seekg(it->start);
        ifs.read(content, it->len);
        SplitToolCppJieba splitTool;
        vector<string> keyTable = splitTool.cut(content);

        //过滤掉暂停词
        filterStopWords(keyTable);

        vector<pair<int, int>> tmpVector;
        for(auto & elem: keyTable){
            tmpVector.clear();
            auto ret = _table.find(elem);
            if(ret == _table.end()){
                tmpVector.push_back(make_pair(it->id, 1));
                _table.insert(make_pair(elem, tmpVector));
            }
            else{
                auto it3 = ret->second.begin();
                for(it3 = ret->second.begin(); it3 != ret->second.end(); ++it3){
                    if(it3->first == it->id){
                        it3->second++;
                        break;
                    }
                }
                if(it3 == ret->second.end()){
                    ret->second.push_back({it->id, 1});
                }
            }
        }

    }

    delete [] content;
    ifs.close();
    return 0;
}
#endif

int InvertedIndex::generateTable(){
    readOffset();

    const char*const DICT_PATH="../include/cppjieba/dict/jieba.dict.utf8";
       const char*const HMM_PATH="../include/cppjieba/dict/hmm_model.utf8";
       const char*const USER_DICT_PATH= "../include/cppjieba/dict/user.dict.utf8";
       const char*const IDF_PATH="../include/cppjieba/dict/idf.utf8";
       const char*const STOP_WORD_PATH="../include/cppjieba/dict/stop_words.utf8";

    cppjieba::Jieba jieba(DICT_PATH,HMM_PATH,USER_DICT_PATH,IDF_PATH,STOP_WORD_PATH);

    //SplitToolCppJieba splitTool;
    //string content;

    int tmpCnt = 0;
    vector<string> keyTable;
    keyTable.reserve(4096);
    
    for(vector<Offset>::iterator it = _offsetVector.begin(); it != _offsetVector.end(); ++it, ++tmpCnt){
        //分割出单词
        keyTable.clear();
        //content.clear();

        //cout << "111111111111" << "  cnt = " << tmpCnt << endl;

        // for(auto it5 = _pageLib.begin(); it5 != _pageLib.end(); ++it5){

        //     cout << "it5->id = " << it5->_docId << endl;
 
        //     if(it5->_docId == it->id){
        //         keyTable = splitTool.cut(it5->_docContent);
        //         break;
        //     }
        // }

        //keyTable = splitTool.cut(_pageLib[tmpCnt]._docContent);
        jieba.Cut(_pageLib[tmpCnt]._docContent, keyTable);

        //cout << "222222222222" << "  cnt = " << tmpCnt << endl;
        
        //过滤掉暂停词
        filterStopWords(keyTable);

        //填充表
        vector<pair<int, int>> tmpVector;
        for(auto & elem: keyTable){
            tmpVector.clear();
            auto ret = _table.find(elem);
            if(ret == _table.end()){
                tmpVector.push_back(make_pair(it->id, 1));
                _table.insert(make_pair(elem, tmpVector));
            }
            else{
                auto it3 = ret->second.begin();
                for(it3 = ret->second.begin(); it3 != ret->second.end(); ++it3){
                    if(it3->first == it->id){
                        it3->second++;
                        break;
                    }
                }
                if(it3 == ret->second.end()){
                    ret->second.push_back({it->id, 1});
                }
            }
        }

        //cout << "33333333333333" << "   cnt = " << tmpCnt << endl;

    }

    return 0;
}


void InvertedIndex::printTable(){
    for (auto it = _table.begin(); it != _table.end(); ++it) {
        cout << it->first << endl;
        for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){
            cout << "    " << it2->first << "  " << it2->second << endl;
        }
    }
    cout << endl;
}

int InvertedIndex::buildInvertedIndexTable(){
    int readFileRet = generateTable();
    if(readFileRet != 0){ 
        return -1;
    }

    vector<pair<int, double>> tempVector;

    //生成全部w，暂存在_invertIndexTable
    for(auto it = _table.begin(); it != _table.end(); ++it){
        tempVector.clear();
        for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){
            double tf, df, idf, w, n;
            tf = it2->second;
            df = it->second.size();
            n = _offsetVector.size();
            idf = (log(n/(df + 1)))/(log(2.0)); //换底公式
            w = tf * idf;
            
            tempVector.push_back({it2->first, w});
        }
        _invertIndexTable.insert(make_pair(it->first, tempVector));
    }

    //计算每一篇文章的 w1^2 + w2^2 + ... + wn^2
    map<int, double> w2SumMap;
    for(auto it = _invertIndexTable.begin(); it != _invertIndexTable.end(); ++it){
        for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){
            w2SumMap[it2->first] += it2->second * it2->second;
        }
    }

    //生成w'，得到最终的倒排索引表
    for(auto it = _invertIndexTable.begin(); it != _invertIndexTable.end(); ++it){
        for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){
            it2->second = it2->second / sqrt(w2SumMap[it2->first]);
        }
    }

    //把倒排索引表输出到文件
     int writeFileRet = writerFile();
     if(writeFileRet != 0){
         return -1;
     }
    return 0;
}

int InvertedIndex::writerFile(){
    ofstream outputFile(_invertedIndexFilePath);
    if(!outputFile){
        cerr << "open file " << _invertedIndexFilePath << " error!" << endl;
        return -1;
    }

    for(auto it = _invertIndexTable.begin(); it != _invertIndexTable.end(); ++it){
        outputFile << it->first << " ";
        for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){
            outputFile << it2->first << " " << it2->second <<" ";
        }
        outputFile << endl;
    }
    outputFile.close();  
    return 0;
}

void InvertedIndex::printInvertedIndexTable(){
    for(auto it = _invertIndexTable.begin(); it != _invertIndexTable.end(); ++it){
        cout << it->first <<endl;
        for(auto it2 = it->second.begin(); it2 != it->second.end(); ++it2){
            cout << "    " << it2->first << "  " << it2->second << endl;
        }
    }
}


